suricata
tm-threads.h
Go to the documentation of this file.
1 /* Copyright (C) 2007-2011 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  * \author Anoop Saldanha <anoopsaldanha@gmail.com>
23  */
24 
25 #ifndef __TM_THREADS_H__
26 #define __TM_THREADS_H__
27 
28 #include "tmqh-packetpool.h"
29 #include "tm-threads-common.h"
30 #include "tm-modules.h"
31 #include "flow.h" // for the FlowQueue
32 
33 #ifdef OS_WIN32
34 static inline void SleepUsec(uint64_t usec)
35 {
36  uint64_t msec = 1;
37  if (usec > 1000) {
38  msec = usec / 1000;
39  }
40  Sleep(msec);
41 }
42 #define SleepMsec(msec) Sleep((msec))
43 #else
44 #define SleepUsec(usec) usleep((usec))
45 #define SleepMsec(msec) usleep((msec) * 1000)
46 #endif
47 
48 #define TM_QUEUE_NAME_MAX 16
49 #define TM_THREAD_NAME_MAX 16
50 
51 typedef TmEcode (*TmSlotFunc)(ThreadVars *, Packet *, void *);
52 
53 typedef struct TmSlot_ {
54  /* function pointers */
55  union {
57  TmEcode (*PktAcqLoop)(ThreadVars *, void *, void *);
58  TmEcode (*Management)(ThreadVars *, void *);
59  };
60  /** linked list of slots, used when a pipeline has multiple slots
61  * in a single thread. */
62  struct TmSlot_ *slot_next;
63 
64  SC_ATOMIC_DECLARE(void *, slot_data);
65 
66  TmEcode (*SlotThreadInit)(ThreadVars *, const void *, void **);
69 
70  /* data storage */
71  const void *slot_initdata;
72  /* store the thread module id */
73  int tm_id;
74 
76 
77 extern ThreadVars *tv_root[TVT_MAX];
78 
79 extern SCMutex tv_root_lock;
80 
81 void TmSlotSetFuncAppend(ThreadVars *, TmModule *, const void *);
82 
83 ThreadVars *TmThreadCreate(const char *, const char *, const char *, const char *, const char *, const char *,
84  void *(fn_p)(void *), int);
85 ThreadVars *TmThreadCreatePacketHandler(const char *, const char *, const char *, const char *, const char *,
86  const char *);
87 ThreadVars *TmThreadCreateMgmtThread(const char *name, void *(fn_p)(void *), int);
88 ThreadVars *TmThreadCreateMgmtThreadByName(const char *name, const char *module,
89  int mucond);
90 ThreadVars *TmThreadCreateCmdThreadByName(const char *name, const char *module,
91  int mucond);
93 void TmThreadSetFlags(ThreadVars *, uint8_t);
94 void TmThreadKillThreadsFamily(int family);
95 void TmThreadKillThreads(void);
96 void TmThreadClearThreadsFamily(int family);
97 void TmThreadAppend(ThreadVars *, int);
98 void TmThreadSetGroupName(ThreadVars *tv, const char *name);
99 
102 TmEcode TmThreadSetCPU(ThreadVars *, uint8_t);
105 int TmThreadGetNbThreads(uint8_t type);
106 
107 void TmThreadInitMC(ThreadVars *);
110 void TmThreadContinueThreads(void);
111 void TmThreadCheckThreadState(void);
113 
114 int TmThreadsCheckFlag(ThreadVars *, uint32_t);
115 void TmThreadsSetFlag(ThreadVars *, uint32_t);
116 void TmThreadsUnsetFlag(ThreadVars *, uint32_t);
117 void TmThreadWaitForFlag(ThreadVars *, uint32_t);
118 
120 
123 
124 uint32_t TmThreadCountThreadsByTmmFlags(uint8_t flags);
125 
127 
128 static inline void TmThreadsCleanDecodePQ(PacketQueueNoLock *pq)
129 {
130  while (1) {
131  Packet *p = PacketDequeueNoLock(pq);
132  if (unlikely(p == NULL))
133  break;
134  TmqhOutputPacketpool(NULL, p);
135  }
136 }
137 
138 static inline void TmThreadsSlotProcessPktFail(ThreadVars *tv, TmSlot *s, Packet *p)
139 {
140  if (p != NULL) {
142  }
143  TmThreadsCleanDecodePQ(&tv->decode_pq);
144  if (tv->stream_pq_local) {
148  }
150 }
151 
152 /**
153  * \brief Handle timeout from the capture layer. Checks
154  * stream_pq which may have been filled by the flow
155  * manager.
156  * \param s pipeline to run on these packets.
157  */
158 static inline bool TmThreadsHandleInjectedPackets(ThreadVars *tv)
159 {
161  if (pq && pq->len > 0) {
162  while (1) {
163  SCMutexLock(&pq->mutex_q);
164  Packet *extra_p = PacketDequeue(pq);
165  SCMutexUnlock(&pq->mutex_q);
166  if (extra_p == NULL)
167  break;
169  if (r == TM_ECODE_FAILED) {
170  TmThreadsSlotProcessPktFail(tv, tv->tm_flowworker, extra_p);
171  break;
172  }
173  tv->tmqh_out(tv, extra_p);
174  }
175  return true;
176  } else {
177  return false;
178  }
179 }
180 
181 /**
182  * \brief Process the rest of the functions (if any) and queue.
183  */
184 static inline TmEcode TmThreadsSlotProcessPkt(ThreadVars *tv, TmSlot *s, Packet *p)
185 {
186  if (s == NULL) {
187  tv->tmqh_out(tv, p);
188  return TM_ECODE_OK;
189  }
190 
191  TmEcode r = TmThreadsSlotVarRun(tv, p, s);
192  if (unlikely(r == TM_ECODE_FAILED)) {
193  TmThreadsSlotProcessPktFail(tv, s, p);
194  return TM_ECODE_FAILED;
195  }
196 
197  tv->tmqh_out(tv, p);
198 
199  TmThreadsHandleInjectedPackets(tv);
200 
201  return TM_ECODE_OK;
202 }
203 
204 /** \brief inject packet if THV_CAPTURE_INJECT_PKT is set
205  * Allow caller to supply their own packet
206  *
207  * Meant for detect reload process that interupts an sleeping capture thread
208  * to force a packet through the engine to complete a reload */
209 static inline void TmThreadsCaptureInjectPacket(ThreadVars *tv, Packet *p)
210 {
212  if (p == NULL)
214  if (p != NULL) {
217  if (TmThreadsSlotProcessPkt(tv, tv->tm_flowworker, p) != TM_ECODE_OK) {
219  }
220  }
221 }
222 
223 /** \brief handle capture timeout
224  * When a capture method times out we check for house keeping
225  * tasks in the capture thread.
226  *
227  * \param p packet. Capture method may have taken a packet from
228  * the pool prior to the timing out call. We will then
229  * use that packet. Otherwise we can get our own.
230  */
231 static inline void TmThreadsCaptureHandleTimeout(ThreadVars *tv, Packet *p)
232 {
234  TmThreadsCaptureInjectPacket(tv, p); /* consumes 'p' */
235  return;
236 
237  } else {
238  if (TmThreadsHandleInjectedPackets(tv) == false) {
239  /* see if we have to do some house keeping */
240  if (tv->flow_queue && SC_ATOMIC_GET(tv->flow_queue->non_empty) == true) {
241  TmThreadsCaptureInjectPacket(tv, p); /* consumes 'p' */
242  return;
243  }
244  }
245  }
246 
247  /* packet could have been passed to us that we won't use
248  * return it to the pool. */
249  if (p != NULL)
250  tv->tmqh_out(tv, p);
251 }
252 
253 void TmThreadsListThreads(void);
254 int TmThreadsRegisterThread(ThreadVars *tv, const int type);
255 void TmThreadsUnregisterThread(const int id);
256 void TmThreadsInjectFlowById(Flow *f, const int id);
257 
258 void TmThreadsInitThreadsTimestamp(const struct timeval *ts);
259 void TmThreadsSetThreadTimestamp(const int id, const struct timeval *ts);
260 void TmThreadsGetMinimalTimestamp(struct timeval *ts);
261 uint16_t TmThreadsGetWorkerThreadMax(void);
262 bool TmThreadsTimeSubsysIsReady(void);
263 
264 #endif /* __TM_THREADS_H__ */
tv_root
ThreadVars * tv_root[TVT_MAX]
Definition: tm-threads.c:79
TmSlot_::tm_id
int tm_id
Definition: tm-threads.h:73
ThreadVars_::flow_queue
struct FlowQueue_ * flow_queue
Definition: threadvars.h:134
ts
uint64_t ts
Definition: source-erf-file.c:55
TmThreadSetPrio
void TmThreadSetPrio(ThreadVars *)
Adjusting nice value for threads.
Definition: tm-threads.c:760
TmThreadContinue
void TmThreadContinue(ThreadVars *)
Unpauses a thread.
Definition: tm-threads.c:1776
TmThreadSetCPU
TmEcode TmThreadSetCPU(ThreadVars *, uint8_t)
Definition: tm-threads.c:805
TmThreadsListThreads
void TmThreadsListThreads(void)
Definition: tm-threads.c:2071
unlikely
#define unlikely(expr)
Definition: util-optimize.h:35
PacketQueue_
simple fifo queue for packets with mutex and cond Calling the mutex or triggering the cond is respons...
Definition: packet-queue.h:49
TmSlot_::SlotFunc
TmSlotFunc SlotFunc
Definition: tm-threads.h:56
PKT_SRC_CAPTURE_TIMEOUT
@ PKT_SRC_CAPTURE_TIMEOUT
Definition: decode.h:64
Packet_::flags
uint32_t flags
Definition: decode.h:460
Flow_
Flow data structure.
Definition: flow.h:356
TmThreadsSlotVarRun
TmEcode TmThreadsSlotVarRun(ThreadVars *tv, Packet *p, TmSlot *slot)
Separate run function so we can call it recursively.
Definition: tm-threads.c:114
TmThreadsInjectFlowById
void TmThreadsInjectFlowById(Flow *f, const int id)
inject a flow into a threads flow queue
Definition: tm-threads.c:2290
tm-threads-common.h
SCMutexLock
#define SCMutexLock(mut)
Definition: threads-debug.h:117
TmThreadKillThreadsFamily
void TmThreadKillThreadsFamily(int family)
Definition: tm-threads.c:1513
ThreadVars_::stream_pq_local
struct PacketQueue_ * stream_pq_local
Definition: threadvars.h:116
TmThreadClearThreadsFamily
void TmThreadClearThreadsFamily(int family)
Definition: tm-threads.c:1614
tm-modules.h
TmThreadSpawn
TmEcode TmThreadSpawn(ThreadVars *)
Spawns a thread associated with the ThreadVars instance tv.
Definition: tm-threads.c:1639
TmThreadsInitThreadsTimestamp
void TmThreadsInitThreadsTimestamp(const struct timeval *ts)
Definition: tm-threads.c:2205
SleepUsec
#define SleepUsec(usec)
Definition: tm-threads.h:44
TmqhOutputPacketpool
void TmqhOutputPacketpool(ThreadVars *t, Packet *p)
Definition: tmqh-packetpool.c:357
TM_ECODE_FAILED
@ TM_ECODE_FAILED
Definition: tm-threads-common.h:83
PacketQueueNoLock_
simple fifo queue for packets
Definition: packet-queue.h:34
tmqh-packetpool.h
TmqhReleasePacketsToPacketPool
void TmqhReleasePacketsToPacketPool(PacketQueue *pq)
Release all the packets in the queue back to the packetpool. Mainly used by threads that have failed,...
Definition: tmqh-packetpool.c:467
TmThreadsGetWorkerThreadMax
uint16_t TmThreadsGetWorkerThreadMax(void)
Definition: tm-threads.c:2255
TM_ECODE_OK
@ TM_ECODE_OK
Definition: tm-threads-common.h:82
TmThreadsUnregisterThread
void TmThreadsUnregisterThread(const int id)
Definition: tm-threads.c:2138
PacketQueue_::mutex_q
SCMutex mutex_q
Definition: packet-queue.h:56
PKT_SET_SRC
#define PKT_SET_SRC(p, src_val)
Definition: decode.h:1055
TmThreadDisablePacketThreads
void TmThreadDisablePacketThreads(void)
Disable all packet threads.
Definition: tm-threads.c:1464
TmThreadsGetMinimalTimestamp
void TmThreadsGetMinimalTimestamp(struct timeval *ts)
Definition: tm-threads.c:2220
TmSlot_::PktAcqLoop
TmEcode(* PktAcqLoop)(ThreadVars *, void *, void *)
Definition: tm-threads.h:57
type
uint8_t type
Definition: decode-icmpv4.h:0
PacketDequeueNoLock
Packet * PacketDequeueNoLock(PacketQueueNoLock *qnl)
Definition: packet-queue.c:206
TmSlot_::Management
TmEcode(* Management)(ThreadVars *, void *)
Definition: tm-threads.h:58
SCMutexUnlock
#define SCMutexUnlock(mut)
Definition: threads-debug.h:119
PKT_PSEUDO_STREAM_END
#define PKT_PSEUDO_STREAM_END
Definition: decode.h:998
ThreadVars_
Per thread variable structure.
Definition: threadvars.h:57
TmThreadDisableReceiveThreads
void TmThreadDisableReceiveThreads(void)
Disable all threads having the specified TMs.
Definition: tm-threads.c:1332
TmSlotSetFuncAppend
void TmSlotSetFuncAppend(ThreadVars *, TmModule *, const void *)
Appends a new entry to the slots.
Definition: tm-threads.c:633
TmThreadCreateCmdThreadByName
ThreadVars * TmThreadCreateCmdThreadByName(const char *name, const char *module, int mucond)
Creates and returns the TV instance for a Command thread (CMD). This function supports only custom sl...
Definition: tm-threads.c:1116
TmThreadSetGroupName
void TmThreadSetGroupName(ThreadVars *tv, const char *name)
Definition: tm-threads.c:1596
Packet_
Definition: decode.h:425
TmThreadsSetFlag
void TmThreadsSetFlag(ThreadVars *, uint32_t)
Set a thread flag.
Definition: tm-threads.c:98
TmThreadWaitOnThreadInit
TmEcode TmThreadWaitOnThreadInit(void)
Used to check if all threads have finished their initialization. On finding an un-initialized thread,...
Definition: tm-threads.c:1932
THV_CAPTURE_INJECT_PKT
#define THV_CAPTURE_INJECT_PKT
Definition: threadvars.h:52
TmThreadCreate
ThreadVars * TmThreadCreate(const char *, const char *, const char *, const char *, const char *, const char *, void *(fn_p)(void *), int)
TmSlot_
Definition: tm-threads.h:53
TmEcode
TmEcode
Definition: tm-threads-common.h:81
TmThreadGetNbThreads
int TmThreadGetNbThreads(uint8_t type)
Definition: tm-threads.c:821
TmThreadsRegisterThread
int TmThreadsRegisterThread(ThreadVars *tv, const int type)
Definition: tm-threads.c:2095
TmModule_
Definition: tm-modules.h:43
TmThreadCreateMgmtThread
ThreadVars * TmThreadCreateMgmtThread(const char *name, void *(fn_p)(void *), int)
Creates and returns the TV instance for a Management thread(MGMT). This function supports only custom...
Definition: tm-threads.c:1055
TmThreadCreateMgmtThreadByName
ThreadVars * TmThreadCreateMgmtThreadByName(const char *name, const char *module, int mucond)
Creates and returns the TV instance for a Management thread(MGMT). This function supports only custom...
Definition: tm-threads.c:1083
TmSlot
struct TmSlot_ TmSlot
tv_root_lock
SCMutex tv_root_lock
Definition: tm-threads.c:82
TmThreadKillThreads
void TmThreadKillThreads(void)
Definition: tm-threads.c:1542
flags
uint8_t flags
Definition: decode-gre.h:0
TmThreadAppend
void TmThreadAppend(ThreadVars *, int)
Appends this TV to tv_root based on its type.
Definition: tm-threads.c:1142
ThreadVars_::tmqh_out
void(* tmqh_out)(struct ThreadVars_ *, struct Packet_ *)
Definition: threadvars.h:105
THV_FAILED
#define THV_FAILED
Definition: threadvars.h:40
ThreadVars_::tm_flowworker
struct TmSlot_ * tm_flowworker
Definition: threadvars.h:100
TmThreadInitMC
void TmThreadInitMC(ThreadVars *)
Initializes the mutex and condition variables for this TV.
Definition: tm-threads.c:1696
PacketQueue_::len
uint32_t len
Definition: packet-queue.h:52
TmSlot_::SlotThreadInit
TmEcode(* SlotThreadInit)(ThreadVars *, const void *, void **)
Definition: tm-threads.h:66
TmThreadWaitForFlag
void TmThreadWaitForFlag(ThreadVars *, uint32_t)
Waits till the specified flag(s) is(are) set. We don't bother if the kill flag has been set or not on...
Definition: tm-threads.c:1762
TmSlot_::slot_initdata
const void * slot_initdata
Definition: tm-threads.h:71
tv
ThreadVars * tv
Definition: fuzz_decodepcapfile.c:32
PacketDequeue
Packet * PacketDequeue(PacketQueue *q)
Definition: packet-queue.c:212
TmThreadSetCPUAffinity
TmEcode TmThreadSetCPUAffinity(ThreadVars *, uint16_t)
Set the thread options (cpu affinity).
Definition: tm-threads.c:796
TmThreadsSetThreadTimestamp
void TmThreadsSetThreadTimestamp(const int id, const struct timeval *ts)
Definition: tm-threads.c:2171
TmSlot_::SlotThreadDeinit
TmEcode(* SlotThreadDeinit)(ThreadVars *, void *)
Definition: tm-threads.h:68
TVT_MAX
@ TVT_MAX
Definition: tm-threads-common.h:92
TmSlot_::SlotThreadExitPrintStats
void(* SlotThreadExitPrintStats)(ThreadVars *, void *)
Definition: tm-threads.h:67
TmSlot_::SC_ATOMIC_DECLARE
SC_ATOMIC_DECLARE(void *, slot_data)
ThreadVars_::decode_pq
PacketQueueNoLock decode_pq
Definition: threadvars.h:111
TmThreadCheckThreadState
void TmThreadCheckThreadState(void)
Used to check the thread for certain conditions of failure.
Definition: tm-threads.c:1908
TmThreadTestThreadUnPaused
void TmThreadTestThreadUnPaused(ThreadVars *)
Tests if the thread represented in the arg has been unpaused or not.
Definition: tm-threads.c:1744
TmSlot_::slot_next
struct TmSlot_ * slot_next
Definition: tm-threads.h:62
TmThreadsCheckFlag
int TmThreadsCheckFlag(ThreadVars *, uint32_t)
Check if a thread flag is set.
Definition: tm-threads.c:90
SC_ATOMIC_GET
#define SC_ATOMIC_GET(name)
Get the value from the atomic variable.
Definition: util-atomic.h:376
TmThreadsUnsetFlag
void TmThreadsUnsetFlag(ThreadVars *, uint32_t)
Unset a thread flag.
Definition: tm-threads.c:106
TmThreadSetThreadPriority
TmEcode TmThreadSetThreadPriority(ThreadVars *, int)
Set the thread options (thread priority).
Definition: tm-threads.c:749
TmThreadWaitOnThreadRunning
TmEcode TmThreadWaitOnThreadRunning(void)
Waits for all threads to be in a running state.
Definition: tm-threads.c:1787
flow.h
TmSlotFunc
TmEcode(* TmSlotFunc)(ThreadVars *, Packet *, void *)
Definition: tm-threads.h:51
TmThreadContinueThreads
void TmThreadContinueThreads(void)
Unpauses all threads present in tv_root.
Definition: tm-threads.c:1891
TmThreadSetFlags
void TmThreadSetFlags(ThreadVars *, uint8_t)
SCMutex
#define SCMutex
Definition: threads-debug.h:114
PacketGetFromQueueOrAlloc
Packet * PacketGetFromQueueOrAlloc(void)
Get a packet. We try to get a packet from the packetpool first, but if that is empty we alloc a packe...
Definition: decode.c:208
TmThreadSetupOptions
TmEcode TmThreadSetupOptions(ThreadVars *)
Set the thread options (cpu affinitythread). Priority should be already set by pthread_create.
Definition: tm-threads.c:837
TmThreadsTimeSubsysIsReady
bool TmThreadsTimeSubsysIsReady(void)
Definition: tm-threads.c:2188
TmThreadCountThreadsByTmmFlags
uint32_t TmThreadCountThreadsByTmmFlags(uint8_t flags)
returns a count of all the threads that match the flag
Definition: tm-threads.c:1993
TmThreadCreatePacketHandler
ThreadVars * TmThreadCreatePacketHandler(const char *, const char *, const char *, const char *, const char *, const char *)
Creates and returns a TV instance for a Packet Processing Thread. This function doesn't support custo...
Definition: tm-threads.c:1026