suricata
tm-threads.h
Go to the documentation of this file.
1 /* Copyright (C) 2007-2011 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  * \author Anoop Saldanha <anoopsaldanha@gmail.com>
23  */
24 
25 #ifndef __TM_THREADS_H__
26 #define __TM_THREADS_H__
27 
28 #include "tmqh-packetpool.h"
29 #include "tm-threads-common.h"
30 #include "tm-modules.h"
31 #include "flow.h" // for the FlowQueue
32 
33 #ifdef OS_WIN32
34 static inline void SleepUsec(uint64_t usec)
35 {
36  uint64_t msec = 1;
37  if (usec > 1000) {
38  msec = usec / 1000;
39  }
40  Sleep(msec);
41 }
42 #define SleepMsec(msec) Sleep((msec))
43 #else
44 #define SleepUsec(usec) usleep((usec))
45 #define SleepMsec(msec) usleep((msec) * 1000)
46 #endif
47 
48 #define TM_QUEUE_NAME_MAX 16
49 #define TM_THREAD_NAME_MAX 16
50 
51 typedef TmEcode (*TmSlotFunc)(ThreadVars *, Packet *, void *);
52 
53 typedef struct TmSlot_ {
54  /* function pointers */
55  union {
57  TmEcode (*PktAcqLoop)(ThreadVars *, void *, void *);
58  TmEcode (*Management)(ThreadVars *, void *);
59  };
60  /** linked list of slots, used when a pipeline has multiple slots
61  * in a single thread. */
62  struct TmSlot_ *slot_next;
63 
64  SC_ATOMIC_DECLARE(void *, slot_data);
65 
66  TmEcode (*SlotThreadInit)(ThreadVars *, const void *, void **);
69 
70  /* data storage */
71  const void *slot_initdata;
72  /* store the thread module id */
73  int tm_id;
74 
76 
77 extern ThreadVars *tv_root[TVT_MAX];
78 
79 extern SCMutex tv_root_lock;
80 
81 void TmSlotSetFuncAppend(ThreadVars *, TmModule *, const void *);
82 
83 ThreadVars *TmThreadCreate(const char *, const char *, const char *, const char *, const char *, const char *,
84  void *(fn_p)(void *), int);
85 ThreadVars *TmThreadCreatePacketHandler(const char *, const char *, const char *, const char *, const char *,
86  const char *);
87 ThreadVars *TmThreadCreateMgmtThread(const char *name, void *(fn_p)(void *), int);
88 ThreadVars *TmThreadCreateMgmtThreadByName(const char *name, const char *module,
89  int mucond);
90 ThreadVars *TmThreadCreateCmdThreadByName(const char *name, const char *module,
91  int mucond);
93 void TmThreadKillThreadsFamily(int family);
94 void TmThreadKillThreads(void);
95 void TmThreadClearThreadsFamily(int family);
96 void TmThreadAppend(ThreadVars *, int);
97 void TmThreadSetGroupName(ThreadVars *tv, const char *name);
98 
101 TmEcode TmThreadSetCPU(ThreadVars *, uint8_t);
104 int TmThreadGetNbThreads(uint8_t type);
105 
106 void TmThreadInitMC(ThreadVars *);
109 void TmThreadContinueThreads(void);
110 void TmThreadCheckThreadState(void);
112 
113 int TmThreadsCheckFlag(ThreadVars *, uint32_t);
114 void TmThreadsSetFlag(ThreadVars *, uint32_t);
115 void TmThreadsUnsetFlag(ThreadVars *, uint32_t);
116 void TmThreadWaitForFlag(ThreadVars *, uint32_t);
117 
119 
122 
123 uint32_t TmThreadCountThreadsByTmmFlags(uint8_t flags);
124 
126 
128  ThreadVars *tv, PacketQueueNoLock *decode_pq, TmSlot *slot);
129 
130 static inline void TmThreadsCleanDecodePQ(PacketQueueNoLock *pq)
131 {
132  while (1) {
133  Packet *p = PacketDequeueNoLock(pq);
134  if (unlikely(p == NULL))
135  break;
136  TmqhOutputPacketpool(NULL, p);
137  }
138 }
139 
140 static inline void TmThreadsSlotProcessPktFail(ThreadVars *tv, TmSlot *s, Packet *p)
141 {
142  if (p != NULL) {
144  }
145  TmThreadsCleanDecodePQ(&tv->decode_pq);
146  if (tv->stream_pq_local) {
150  }
152 }
153 
154 /**
155  * \brief Handle timeout from the capture layer. Checks
156  * stream_pq which may have been filled by the flow
157  * manager.
158  * \param s pipeline to run on these packets.
159  */
160 static inline bool TmThreadsHandleInjectedPackets(ThreadVars *tv)
161 {
163  if (pq && pq->len > 0) {
164  while (1) {
165  SCMutexLock(&pq->mutex_q);
166  Packet *extra_p = PacketDequeue(pq);
167  SCMutexUnlock(&pq->mutex_q);
168  if (extra_p == NULL)
169  break;
170 #ifdef DEBUG_VALIDATION
171  BUG_ON(extra_p->flow != NULL);
172 #endif
174  if (r == TM_ECODE_FAILED) {
175  TmThreadsSlotProcessPktFail(tv, tv->tm_flowworker, extra_p);
176  break;
177  }
178  tv->tmqh_out(tv, extra_p);
179  }
180  return true;
181  } else {
182  return false;
183  }
184 }
185 
186 /**
187  * \brief Process the rest of the functions (if any) and queue.
188  */
189 static inline TmEcode TmThreadsSlotProcessPkt(ThreadVars *tv, TmSlot *s, Packet *p)
190 {
191  if (s == NULL) {
192  tv->tmqh_out(tv, p);
193  return TM_ECODE_OK;
194  }
195 
196  TmEcode r = TmThreadsSlotVarRun(tv, p, s);
197  if (unlikely(r == TM_ECODE_FAILED)) {
198  TmThreadsSlotProcessPktFail(tv, s, p);
199  return TM_ECODE_FAILED;
200  }
201 
202  tv->tmqh_out(tv, p);
203 
204  TmThreadsHandleInjectedPackets(tv);
205 
206  return TM_ECODE_OK;
207 }
208 
209 /** \brief inject packet if THV_CAPTURE_INJECT_PKT is set
210  * Allow caller to supply their own packet
211  *
212  * Meant for detect reload process that interrupts an sleeping capture thread
213  * to force a packet through the engine to complete a reload */
214 static inline void TmThreadsCaptureInjectPacket(ThreadVars *tv, Packet *p)
215 {
217  if (p == NULL)
219  if (p != NULL) {
222  if (TmThreadsSlotProcessPkt(tv, tv->tm_flowworker, p) != TM_ECODE_OK) {
224  }
225  }
226 }
227 
228 /** \brief handle capture timeout
229  * When a capture method times out we check for house keeping
230  * tasks in the capture thread.
231  *
232  * \param p packet. Capture method may have taken a packet from
233  * the pool prior to the timing out call. We will then
234  * use that packet. Otherwise we can get our own.
235  */
236 static inline void TmThreadsCaptureHandleTimeout(ThreadVars *tv, Packet *p)
237 {
239  TmThreadsCaptureInjectPacket(tv, p); /* consumes 'p' */
240  return;
241 
242  } else {
243  if (TmThreadsHandleInjectedPackets(tv) == false) {
244  /* see if we have to do some house keeping */
245  if (tv->flow_queue && SC_ATOMIC_GET(tv->flow_queue->non_empty) == true) {
246  TmThreadsCaptureInjectPacket(tv, p); /* consumes 'p' */
247  return;
248  }
249  }
250  }
251 
252  /* packet could have been passed to us that we won't use
253  * return it to the pool. */
254  if (p != NULL)
255  tv->tmqh_out(tv, p);
256 }
257 
258 static inline void TmThreadsCaptureBreakLoop(ThreadVars *tv)
259 {
260  if ((tv->tmm_flags & TM_FLAG_RECEIVE_TM) == 0) {
261  return;
262  }
263  /* find the correct slot */
264  TmSlot *s = tv->tm_slots;
265  TmModule *tm = TmModuleGetById(s->tm_id);
266  if (tm->flags & TM_FLAG_RECEIVE_TM) {
267  /* if the method supports it, BreakLoop. Otherwise we rely on
268  * the capture method's recv timeout */
269  if (tm->PktAcqLoop && tm->PktAcqBreakLoop) {
270  tm->PktAcqBreakLoop(tv, SC_ATOMIC_GET(s->slot_data));
271  }
273  }
274 }
275 
276 void TmThreadsListThreads(void);
277 int TmThreadsRegisterThread(ThreadVars *tv, const int type);
278 void TmThreadsUnregisterThread(const int id);
279 void TmThreadsInjectFlowById(Flow *f, const int id);
280 
282 void TmThreadsSetThreadTimestamp(const int id, const SCTime_t ts);
283 void TmThreadsGetMinimalTimestamp(struct timeval *ts);
284 uint16_t TmThreadsGetWorkerThreadMax(void);
285 bool TmThreadsTimeSubsysIsReady(void);
286 
287 #endif /* __TM_THREADS_H__ */
tv_root
ThreadVars * tv_root[TVT_MAX]
Definition: tm-threads.c:80
TmSlot_::tm_id
int tm_id
Definition: tm-threads.h:73
ThreadVars_::flow_queue
struct FlowQueue_ * flow_queue
Definition: threadvars.h:134
ts
uint64_t ts
Definition: source-erf-file.c:55
TmThreadsInitThreadsTimestamp
void TmThreadsInitThreadsTimestamp(const SCTime_t ts)
Definition: tm-threads.c:2214
TmThreadSetPrio
void TmThreadSetPrio(ThreadVars *)
Adjusting nice value for threads.
Definition: tm-threads.c:768
TmThreadContinue
void TmThreadContinue(ThreadVars *)
Unpauses a thread.
Definition: tm-threads.c:1785
TmThreadSetCPU
TmEcode TmThreadSetCPU(ThreadVars *, uint8_t)
Definition: tm-threads.c:814
TmThreadsListThreads
void TmThreadsListThreads(void)
Definition: tm-threads.c:2080
unlikely
#define unlikely(expr)
Definition: util-optimize.h:35
PacketQueue_
simple fifo queue for packets with mutex and cond Calling the mutex or triggering the cond is respons...
Definition: packet-queue.h:49
TmSlot_::SlotFunc
TmSlotFunc SlotFunc
Definition: tm-threads.h:56
PKT_SRC_CAPTURE_TIMEOUT
@ PKT_SRC_CAPTURE_TIMEOUT
Definition: decode.h:64
Packet_::flags
uint32_t flags
Definition: decode.h:467
Flow_
Flow data structure.
Definition: flow.h:347
TmThreadsSlotVarRun
TmEcode TmThreadsSlotVarRun(ThreadVars *tv, Packet *p, TmSlot *slot)
Separate run function so we can call it recursively.
Definition: tm-threads.c:131
TmThreadsInjectFlowById
void TmThreadsInjectFlowById(Flow *f, const int id)
inject a flow into a threads flow queue
Definition: tm-threads.c:2283
tm-threads-common.h
SCMutexLock
#define SCMutexLock(mut)
Definition: threads-debug.h:117
TmThreadKillThreadsFamily
void TmThreadKillThreadsFamily(int family)
Definition: tm-threads.c:1525
ThreadVars_::stream_pq_local
struct PacketQueue_ * stream_pq_local
Definition: threadvars.h:116
TmThreadClearThreadsFamily
void TmThreadClearThreadsFamily(int family)
Definition: tm-threads.c:1626
tm-modules.h
TmThreadSpawn
TmEcode TmThreadSpawn(ThreadVars *)
Spawns a thread associated with the ThreadVars instance tv.
Definition: tm-threads.c:1651
SleepUsec
#define SleepUsec(usec)
Definition: tm-threads.h:44
TmqhOutputPacketpool
void TmqhOutputPacketpool(ThreadVars *t, Packet *p)
Definition: tmqh-packetpool.c:356
TM_ECODE_FAILED
@ TM_ECODE_FAILED
Definition: tm-threads-common.h:85
TmThreadsProcessDecodePseudoPackets
TmEcode TmThreadsProcessDecodePseudoPackets(ThreadVars *tv, PacketQueueNoLock *decode_pq, TmSlot *slot)
Definition: tm-threads.c:112
PacketQueueNoLock_
simple fifo queue for packets
Definition: packet-queue.h:34
tmqh-packetpool.h
TmqhReleasePacketsToPacketPool
void TmqhReleasePacketsToPacketPool(PacketQueue *pq)
Release all the packets in the queue back to the packetpool. Mainly used by threads that have failed,...
Definition: tmqh-packetpool.c:471
TmThreadsGetWorkerThreadMax
uint16_t TmThreadsGetWorkerThreadMax(void)
Definition: tm-threads.c:2265
TmModule_::PktAcqLoop
TmEcode(* PktAcqLoop)(ThreadVars *, void *, void *)
Definition: tm-modules.h:54
TM_ECODE_OK
@ TM_ECODE_OK
Definition: tm-threads-common.h:84
TmThreadsUnregisterThread
void TmThreadsUnregisterThread(const int id)
Definition: tm-threads.c:2147
PacketQueue_::mutex_q
SCMutex mutex_q
Definition: packet-queue.h:56
PKT_SET_SRC
#define PKT_SET_SRC(p, src_val)
Definition: decode.h:1056
TmThreadDisablePacketThreads
void TmThreadDisablePacketThreads(void)
Disable all packet threads.
Definition: tm-threads.c:1476
ThreadVars_::tmm_flags
uint8_t tmm_flags
Definition: threadvars.h:78
TmThreadsGetMinimalTimestamp
void TmThreadsGetMinimalTimestamp(struct timeval *ts)
Definition: tm-threads.c:2229
TmThreadsSetThreadTimestamp
void TmThreadsSetThreadTimestamp(const int id, const SCTime_t ts)
Definition: tm-threads.c:2180
TmSlot_::PktAcqLoop
TmEcode(* PktAcqLoop)(ThreadVars *, void *, void *)
Definition: tm-threads.h:57
type
uint8_t type
Definition: decode-icmpv4.h:0
TmModule_::PktAcqBreakLoop
TmEcode(* PktAcqBreakLoop)(ThreadVars *, void *)
Definition: tm-modules.h:57
ThreadVars_::tm_slots
struct TmSlot_ * tm_slots
Definition: threadvars.h:95
PacketDequeueNoLock
Packet * PacketDequeueNoLock(PacketQueueNoLock *qnl)
Definition: packet-queue.c:208
TmSlot_::Management
TmEcode(* Management)(ThreadVars *, void *)
Definition: tm-threads.h:58
SCMutexUnlock
#define SCMutexUnlock(mut)
Definition: threads-debug.h:119
PKT_PSEUDO_STREAM_END
#define PKT_PSEUDO_STREAM_END
Definition: decode.h:1002
ThreadVars_
Per thread variable structure.
Definition: threadvars.h:57
TmThreadDisableReceiveThreads
void TmThreadDisableReceiveThreads(void)
Disable all threads having the specified TMs.
Definition: tm-threads.c:1348
TmSlotSetFuncAppend
void TmSlotSetFuncAppend(ThreadVars *, TmModule *, const void *)
Appends a new entry to the slots.
Definition: tm-threads.c:640
BUG_ON
#define BUG_ON(x)
Definition: suricata-common.h:295
TmThreadCreateCmdThreadByName
ThreadVars * TmThreadCreateCmdThreadByName(const char *name, const char *module, int mucond)
Creates and returns the TV instance for a Command thread (CMD). This function supports only custom sl...
Definition: tm-threads.c:1125
TmThreadSetGroupName
void TmThreadSetGroupName(ThreadVars *tv, const char *name)
Definition: tm-threads.c:1608
Packet_
Definition: decode.h:430
TmThreadsSetFlag
void TmThreadsSetFlag(ThreadVars *, uint32_t)
Set a thread flag.
Definition: tm-threads.c:99
TmThreadWaitOnThreadInit
TmEcode TmThreadWaitOnThreadInit(void)
Used to check if all threads have finished their initialization. On finding an un-initialized thread,...
Definition: tm-threads.c:1939
THV_CAPTURE_INJECT_PKT
#define THV_CAPTURE_INJECT_PKT
Definition: threadvars.h:52
TmThreadCreate
ThreadVars * TmThreadCreate(const char *, const char *, const char *, const char *, const char *, const char *, void *(fn_p)(void *), int)
TmModuleGetById
TmModule * TmModuleGetById(int id)
Returns a TM Module by its id.
Definition: tm-modules.c:90
TmSlot_
Definition: tm-threads.h:53
SCTime_t
Definition: util-time.h:40
TmEcode
TmEcode
Definition: tm-threads-common.h:83
TmThreadGetNbThreads
int TmThreadGetNbThreads(uint8_t type)
Definition: tm-threads.c:830
TM_FLAG_RECEIVE_TM
#define TM_FLAG_RECEIVE_TM
Definition: tm-modules.h:31
TmThreadsRegisterThread
int TmThreadsRegisterThread(ThreadVars *tv, const int type)
Definition: tm-threads.c:2104
TmModule_
Definition: tm-modules.h:43
TmThreadCreateMgmtThread
ThreadVars * TmThreadCreateMgmtThread(const char *name, void *(fn_p)(void *), int)
Creates and returns the TV instance for a Management thread(MGMT). This function supports only custom...
Definition: tm-threads.c:1064
TmThreadCreateMgmtThreadByName
ThreadVars * TmThreadCreateMgmtThreadByName(const char *name, const char *module, int mucond)
Creates and returns the TV instance for a Management thread(MGMT). This function supports only custom...
Definition: tm-threads.c:1092
TmSlot
struct TmSlot_ TmSlot
tv_root_lock
SCMutex tv_root_lock
Definition: tm-threads.c:83
Packet_::flow
struct Flow_ * flow
Definition: decode.h:469
TmThreadKillThreads
void TmThreadKillThreads(void)
Definition: tm-threads.c:1554
flags
uint8_t flags
Definition: decode-gre.h:0
TmThreadAppend
void TmThreadAppend(ThreadVars *, int)
Appends this TV to tv_root based on its type.
Definition: tm-threads.c:1151
ThreadVars_::tmqh_out
void(* tmqh_out)(struct ThreadVars_ *, struct Packet_ *)
Definition: threadvars.h:105
THV_FAILED
#define THV_FAILED
Definition: threadvars.h:40
ThreadVars_::tm_flowworker
struct TmSlot_ * tm_flowworker
Definition: threadvars.h:100
TmThreadInitMC
void TmThreadInitMC(ThreadVars *)
Initializes the mutex and condition variables for this TV.
Definition: tm-threads.c:1707
PacketQueue_::len
uint32_t len
Definition: packet-queue.h:52
TmSlot_::SlotThreadInit
TmEcode(* SlotThreadInit)(ThreadVars *, const void *, void **)
Definition: tm-threads.h:66
TmThreadWaitForFlag
void TmThreadWaitForFlag(ThreadVars *, uint32_t)
Waits till the specified flag(s) is(are) set. We don't bother if the kill flag has been set or not on...
Definition: tm-threads.c:1771
TmSlot_::slot_initdata
const void * slot_initdata
Definition: tm-threads.h:71
tv
ThreadVars * tv
Definition: fuzz_decodepcapfile.c:32
PacketDequeue
Packet * PacketDequeue(PacketQueue *q)
Definition: packet-queue.c:216
TmThreadSetCPUAffinity
TmEcode TmThreadSetCPUAffinity(ThreadVars *, uint16_t)
Set the thread options (cpu affinity).
Definition: tm-threads.c:805
TVT_MAX
@ TVT_MAX
Definition: tm-threads-common.h:94
TmSlot_::SlotThreadDeinit
TmEcode(* SlotThreadDeinit)(ThreadVars *, void *)
Definition: tm-threads.h:68
TmSlot_::SlotThreadExitPrintStats
void(* SlotThreadExitPrintStats)(ThreadVars *, void *)
Definition: tm-threads.h:67
TmSlot_::SC_ATOMIC_DECLARE
SC_ATOMIC_DECLARE(void *, slot_data)
ThreadVars_::decode_pq
PacketQueueNoLock decode_pq
Definition: threadvars.h:111
TmThreadCheckThreadState
void TmThreadCheckThreadState(void)
Used to check the thread for certain conditions of failure.
Definition: tm-threads.c:1915
TmThreadTestThreadUnPaused
void TmThreadTestThreadUnPaused(ThreadVars *)
Tests if the thread represented in the arg has been unpaused or not.
Definition: tm-threads.c:1753
TmSlot_::slot_next
struct TmSlot_ * slot_next
Definition: tm-threads.h:62
TmThreadsCheckFlag
int TmThreadsCheckFlag(ThreadVars *, uint32_t)
Check if a thread flag is set.
Definition: tm-threads.c:91
SC_ATOMIC_GET
#define SC_ATOMIC_GET(name)
Get the value from the atomic variable.
Definition: util-atomic.h:376
TmThreadsUnsetFlag
void TmThreadsUnsetFlag(ThreadVars *, uint32_t)
Unset a thread flag.
Definition: tm-threads.c:107
TmThreadSetThreadPriority
TmEcode TmThreadSetThreadPriority(ThreadVars *, int)
Set the thread options (thread priority).
Definition: tm-threads.c:757
TmThreadWaitOnThreadRunning
TmEcode TmThreadWaitOnThreadRunning(void)
Waits for all threads to be in a running state.
Definition: tm-threads.c:1796
flow.h
TmSlotFunc
TmEcode(* TmSlotFunc)(ThreadVars *, Packet *, void *)
Definition: tm-threads.h:51
TmThreadContinueThreads
void TmThreadContinueThreads(void)
Unpauses all threads present in tv_root.
Definition: tm-threads.c:1898
SCMutex
#define SCMutex
Definition: threads-debug.h:114
PacketGetFromQueueOrAlloc
Packet * PacketGetFromQueueOrAlloc(void)
Get a packet. We try to get a packet from the packetpool first, but if that is empty we alloc a packe...
Definition: decode.c:208
TmModule_::flags
uint8_t flags
Definition: tm-modules.h:76
TmThreadSetupOptions
TmEcode TmThreadSetupOptions(ThreadVars *)
Set the thread options (cpu affinitythread). Priority should be already set by pthread_create.
Definition: tm-threads.c:846
TmThreadsTimeSubsysIsReady
bool TmThreadsTimeSubsysIsReady(void)
Definition: tm-threads.c:2197
TmThreadCountThreadsByTmmFlags
uint32_t TmThreadCountThreadsByTmmFlags(uint8_t flags)
returns a count of all the threads that match the flag
Definition: tm-threads.c:2002
TmThreadCreatePacketHandler
ThreadVars * TmThreadCreatePacketHandler(const char *, const char *, const char *, const char *, const char *, const char *)
Creates and returns a TV instance for a Packet Processing Thread. This function doesn't support custo...
Definition: tm-threads.c:1035